import requests
from LogManager import log_output
import json
import os

if __name__ == '__main__':
    log_output('获取数据：数据获取中', send_status=True)
    conf = json.load(open('conf.json', 'rt'))
    target_url = conf['target_url']
    result = json.loads(requests.get(target_url+'upload_list').text)

    for file in result['files']:
        file_url = target_url + file
        filename = file.split('/')[-1]
        log_output(f'获取数据：获取{filename}', send_status=True)
        r = requests.get(file_url, stream=True)
        total_length = (int(r.headers.get('content-length')) / 1024) // 1024

        with open(os.path.join('datasets', filename), "wb") as f:
            cnt = 0
            for chunk in r.iter_content(chunk_size=1024):
                if chunk:
                    f.write(chunk)
                cnt += 1
                if cnt % 1024 == 0:
                    log_output(f'获取数据：获取{filename} ({cnt//1024}/{total_length}M已获取)', send_status=True)
